{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "bidi-gru-timeseries-withdropout.ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "TPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/lmoroney/tfbook/blob/master/chapter11/bidi-gru_timeseries-withdropout.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zX4Kg8DUTKWO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
        "# you may not use this file except in compliance with the License.\n",
        "# You may obtain a copy of the License at\n",
        "#\n",
        "# https://www.apache.org/licenses/LICENSE-2.0\n",
        "#\n",
        "# Unless required by applicable law or agreed to in writing, software\n",
        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
        "# See the License for the specific language governing permissions and\n",
        "# limitations under the License."
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D1J15Vh_1Jih",
        "colab_type": "code",
        "cellView": "both",
        "colab": {}
      },
      "source": [
        "try:\n",
        "  # %tensorflow_version only exists in Colab.\n",
        "  %tensorflow_version 2.x\n",
        "except Exception:\n",
        "  pass\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BOjujz601HcS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import tensorflow as tf\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "print(tf.__version__)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "12Ak2nahiFcO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!wget --no-check-certificate \\\n",
        "    https://storage.googleapis.com/learning-datasets/station.csv \\\n",
        "    -O /tmp/station.csv"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MZigDIolhmDS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_data():\n",
        "    data_file = \"/tmp/station.csv\"\n",
        "    f = open(data_file)\n",
        "    data = f.read()\n",
        "    f.close()\n",
        "    lines = data.split('\\n')\n",
        "    header = lines[0].split(',')\n",
        "    lines = lines[1:]\n",
        "    temperatures=[]\n",
        "    for line in lines:\n",
        "        if line:\n",
        "            linedata = line.split(',')\n",
        "            linedata = linedata[1:13]\n",
        "            for item in linedata:\n",
        "                if item:\n",
        "                    temperatures.append(float(item))\n",
        "\n",
        "    series = np.asarray(temperatures)\n",
        "    time = np.arange(len(temperatures), dtype=\"float32\")      \n",
        "    return time, series\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Zswl7jRtGzkk",
        "colab": {}
      },
      "source": [
        "def plot_series(time, series, format=\"-\", start=0, end=None):\n",
        "    plt.plot(time[start:end], series[start:end], format)\n",
        "    plt.xlabel(\"Time\")\n",
        "    plt.ylabel(\"Value\")\n",
        "    plt.grid(True)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kTN-110AhvbN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "time, series = get_data()\n",
        "mean = series.mean(axis=0)\n",
        "series-=mean\n",
        "std = series.std(axis=0)\n",
        "series/=std\n",
        "split_time = 780\n",
        "time_train = time[:split_time]\n",
        "x_train = series[:split_time]\n",
        "time_valid = time[split_time:]\n",
        "x_valid = series[split_time:]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "qR-PmG-RMAX9",
        "colab": {}
      },
      "source": [
        "plt.figure(figsize=(10, 6))\n",
        "plot_series(time_valid, x_valid)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4sTTIOCbyShY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def windowed_dataset(series, window_size, batch_size, shuffle_buffer):\n",
        "  series = tf.expand_dims(series, axis=-1)\n",
        "  dataset = tf.data.Dataset.from_tensor_slices(series)\n",
        "  dataset = dataset.window(window_size + 1, shift=1, drop_remainder=True)\n",
        "  dataset = dataset.flat_map(lambda window: window.batch(window_size + 1))\n",
        "  dataset = dataset.shuffle(shuffle_buffer).map(lambda window: (window[:-1], window[-1]))\n",
        "  dataset = dataset.batch(batch_size).prefetch(1)\n",
        "  return dataset"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KrcN1prrw910",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tf.keras.backend.clear_session()\n",
        "tf.random.set_seed(42)\n",
        "np.random.seed(42)\n",
        "# Experiment with this to play with accuracy. Do note that\n",
        "# this is a small dataset, so a large window size may\n",
        "# cause errors.\n",
        "window_size = 40\n",
        "batch_size = 20\n",
        "shuffle_buffer_size = 20\n",
        "\n",
        "dataset = windowed_dataset(x_train, window_size, batch_size, shuffle_buffer_size)\n",
        "valid_dataset = windowed_dataset(x_valid, window_size, batch_size, shuffle_buffer_size)\n",
        "\n",
        "model = tf.keras.models.Sequential([\n",
        "  tf.keras.layers.Bidirectional(tf.keras.layers.GRU(100, input_shape=[None, 1], return_sequences=True, dropout=0.1, recurrent_dropout=0.1)),\n",
        "  tf.keras.layers.Bidirectional(tf.keras.layers.GRU(100, dropout=0.1, recurrent_dropout=0.1)), \n",
        "  tf.keras.layers.Dense(1),\n",
        "])\n",
        "\n",
        "optimizer = tf.keras.optimizers.SGD(lr=1e-4, momentum=0.7)\n",
        "model.compile(loss=tf.keras.losses.Huber(), optimizer=optimizer, metrics=[\"mae\"])\n",
        "history = model.fit(dataset, epochs=500,  verbose=1, validation_data=valid_dataset)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "_eaAX9g_jS5W",
        "colab": {}
      },
      "source": [
        "def model_forecast(model, series, window_size):\n",
        "    series = tf.expand_dims(series, axis=-1) \n",
        "    ds = tf.data.Dataset.from_tensor_slices(series)\n",
        "    ds = ds.window(window_size, shift=1, drop_remainder=True)\n",
        "    ds = ds.flat_map(lambda w: w.batch(window_size))\n",
        "    ds = ds.batch(32).prefetch(1)\n",
        "    forecast = model.predict(ds)\n",
        "    return forecast"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MKkic-mLdkRZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "forecast = model_forecast(model, series[split_time - window_size: -1], window_size)[:,0]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aa6tNADp5E12",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "plt.figure(figsize=(10,6))\n",
        "plot_series(time_valid, x_valid)\n",
        "plot_series(time_valid, forecast)\n",
        "plt.show()\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-kT6j186YO6K",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "mae = tf.keras.metrics.mean_absolute_error(x_valid, forecast).numpy()\n",
        "print(mae)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5uzk7BCljuW0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "mae=history.history['mae']\n",
        "loss=history.history['loss']\n",
        "val_mae=history.history['val_mae']\n",
        "val_loss=history.history['val_loss']\n",
        "epochs=range(len(loss)) # Get number of epochs\n",
        "\n",
        "#------------------------------------------------\n",
        "# Plot MAE and Loss\n",
        "#------------------------------------------------\n",
        "#plt.plot(epochs, mae, 'r')\n",
        "plt.plot(epochs, loss, 'b')\n",
        "#plt.plot(epochs, val_mae, 'g')\n",
        "plt.plot(epochs, val_loss, 'k')\n",
        "plt.title('Training and Validation Loss')\n",
        "plt.xlabel(\"Epochs\")\n",
        "plt.ylabel(\"Accuracy\")\n",
        "plt.legend([\"Loss\", \"Validation Loss\"])\n",
        "\n",
        "plt.figure()\n",
        "plt.show()\n"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}